//==============================================================================================================================================================
// Author(s): Roman Khmelichek Wei Jiang
//
// TODO: DOCNO can be determined algorithmically from the docID (instead of storing as a string)? We also have to consider docID remapping.
// TODO: Need to implement modes 'kSingleDoc' and 'kStandard'.
//
// Some WARC records in ClueWeb09 are malformed. See 'http://www.umiacs.umd.edu/~jimmylin/cloud9/docs/content/clue.html' for more information. Here, we handle the
// common problem of having an extra new line in the WARC header. A few of the records also happen to have garbled URLs.
//==============================================================================================================================================================

#ifndef PARSERINL_H_
#define PARSERINL_H_

// Enables debugging output for this module.
// #define PARSERINL_DEBUG

#include <cstdlib>
#include <cstring>
#include <fstream>
#include <iostream>
#include <map>
#include <string>

#include "config_file_properties.h"
#include "configuration.h"

using namespace std;

template<class Callback>
  Parser<Callback>::Parser(const Parser<Callback>::ParsingMode& parsing_mode, const Parser<Callback>::DocType& doc_type, Callback* callback) :
    parsing_mode_(parsing_mode), doc_type_(doc_type), callback_(callback) {
    assert(callback_ != NULL);

	// load the needed documents for the phase2 pruning
	LoadUpGov2DocumentSetNeededToBeExtracted();
  }

template<class Callback>
  int Parser<Callback>::LoadClueweb2009SpamReportDict(ifstream &inputfileForDocument) {
	return 0;
}

template<class Callback>
  void Parser<Callback>::LoadUpGov2DocumentSetNeededToBeExtracted(){
	cout << "Updated by Wei on 2013/11/02 night at school" << endl;
	cout << "Load up document sets which needed to parse" << endl;
	string inputFileName = Configuration::GetResultValue( Configuration::GetConfiguration().GetStringValue(config_properties::kGov2DocumentNeededToBeFurtherProcessFileName));
	ifstream inputFileHandler;
	inputFileHandler.open(inputFileName.c_str());

	string currentLine;
    string docID;
    string trecID;
    string mark;
	while ( inputFileHandler.good() )
	{
		getline (inputFileHandler,currentLine);
		boost::algorithm::trim(currentLine);
		istringstream iss( currentLine );

		iss >> docID;
		iss >> trecID;
		iss >> mark;

		if(priority_document_needed_to_process_dict_.count(trecID) > 0){
			// already has this document so just do NOTHING
		}
		else{
			priority_document_needed_to_process_dict_[trecID] = 1;
		}
	}
	inputFileHandler.close();

	// check point
	if (priority_document_needed_to_process_dict_.size() == 0){
		GetDefaultLogger().Log( Stringify(priority_document_needed_to_process_dict_.size() ) + " documents have been loaded into the dict.  --- Take Care", false);
	}
	else{
		GetDefaultLogger().Log( Stringify(priority_document_needed_to_process_dict_.size() ) + " documents have been loaded into the dict.", false);
	}
}

template<class Callback>
  int Parser<Callback>::ShowContentOfSpecifcDocument(const char* buf, int buf_len,const long beginningPosition, const long endingPosition,const char* term, int term_len){
	string lookUpTerm = "";
	string currentWord = "";
	for(int tempCounter = 0; tempCounter < term_len; tempCounter++){
		lookUpTerm += term[tempCounter];
	}
	cout << "lookUpTerm:" << lookUpTerm << endl;

	/*
	cout << "*********************web page content begin" << endl;
	for(int tempCounter = 0; tempCounter < buf_len; tempCounter++){
		cout << buf[tempCounter];
	}
	cout << "*********************web page content End." << endl;
	*/

	cout << "*********************positions & contexts begin" << endl;

	const char* curr_p = buf;

    assert(buf != NULL);
    assert(buf_len > 0);
    uint32_t doc_id = 0;

    Tag tag_ret;  // The special type of tag we encountered.
    unsigned char context = '\0';  // Bit array for the context.
    uint32_t position = 0;         // Tracks position of each word, final position for a document is it's size in words.

    // For parsing HTML.
    bool in_closing_tag = false;  // True when we're parsing a closing tag.
    bool in_script = false;       // True when we're parsing contents of script tag.

    // For TREC documents.
    bool in_doc = false;     // True when we're parsing contents of doc tag.
    bool in_docno = false;   // True when we're parsing contents of docno tag.
    bool in_dochdr = false;  // True when we're parsing contents of dochdr tag.

    // Track the starting point of various things we want to parse out.
    const char* word_p;        // Standalone word.
    //const char* url_p;         // TREC document URL.
    //const char* docno_p;       // TREC document number.
    const char* tag_p = NULL;  // Tracks the starting point of a tag; doubles as a flag as to whether we're currently in a tag.

    //variables set but currently do not use
    if(false){
    	cout << in_doc << endl;
    	cout << in_docno << endl;
    	cout << in_dochdr << endl;
    }

    // The main parsing loop
    while (IsWithinBounds(curr_p, buf, buf_len)) {
      if (!IsIndexable(*curr_p)) {
        if (*curr_p != '>') {
          if (*curr_p == '<') {
            tag_p = curr_p;
          }
          ++curr_p;
          continue;
        }

        if (!tag_p) {
          ++curr_p;
          continue;
        }

        // At this point, we must have just seen the end of a closing tag, '>'.
        ++curr_p;
        tag_ret = ProcessTag(tag_p, curr_p - tag_p, in_closing_tag, doc_id);

        switch (tag_ret) {
          case kTagNot:
            break;

          case kTagB:
            UpdateContext(context, in_closing_tag, kContextB);
            break;

          case kTagI:
            UpdateContext(context, in_closing_tag, kContextI);
            break;

          case kTagH:
            UpdateContext(context, in_closing_tag, kContextH);
            break;

          case kTagTitle:
            UpdateContext(context, in_closing_tag, kContextT);
            break;

          case kTagScript:
            in_script = in_closing_tag ? false : true;
            break;

          //If this tag is to be </Html>.
          case kTagHtml:
            if (doc_type_ != kWarc)
              break;

            if (in_closing_tag) {
              in_doc = false;



              break;
            }

          //If this tag is to be </Doc>.
          case kTagDoc:
            if (doc_type_ != kWarc)
              break;

            if (in_closing_tag) {
              in_doc = false;







            }
            else {
              in_doc = true;
            }
            break;

          case kTagDocno:
            if (doc_type_ != kTrec)
              break;

            in_docno = in_closing_tag ? false : true;
            break;

          case kTagDochdr:
            if (doc_type_ != kTrec)
              break;

            in_dochdr = in_closing_tag ? false : true;
            break;

          default:
            break;
        }

        tag_p = NULL;
        continue;
      }
      // Ignore everything between <script></script> tags and ignore inner contents of tags.
      if (in_script || tag_p) {
        ++curr_p;
        continue;
      }
      word_p = curr_p;
      while (IsWithinBounds(curr_p, buf, buf_len) && IsIndexable(*curr_p)) {
        ++curr_p;
      }

      //callback_->ProcessTerm(word_p, curr_p - word_p, doc_id, position++, context);
      currentWord = "";
      for(int tempCounter = 0; tempCounter < curr_p - word_p; tempCounter++){
    	  currentWord += word_p[tempCounter];
      }
      if(lookUpTerm == currentWord){
    	  cout << lookUpTerm << " " << position++ << " " << int(context) << endl;
      }
      else{
    	  position++;
      }
    }
    cout << "*********************positions & contexts end." << endl;
	return 0;
}

template<class Callback>
  int Parser<Callback>::ShowContentOfSpecifcDocument(const char* buf, int buf_len,const long beginningPosition, const long endingPosition,vector<string> &queryID_Term_docIDList, string currentDocID, ofstream &outputFileHandler){
	//cout << "length:" << queryID_Term_docIDList.size() << endl;

	map<string, string> lookUpTermDocIDDict;
	map<string, vector<int> > outputDict;
	map<string, vector<int> >::iterator outputDictIter;
	string currentWord = "";
	cout << "Looking for the following pairs:" << endl;
	for(unsigned int tempCounter = 0; tempCounter < queryID_Term_docIDList.size(); tempCounter++){
		vector<string> elementsForTheLine;
	    boost::algorithm::split(elementsForTheLine, queryID_Term_docIDList[tempCounter], boost::algorithm::is_any_of("_") );
		lookUpTermDocIDDict[ elementsForTheLine[1] ] = elementsForTheLine[0];
		outputDict[ elementsForTheLine[0] + " " + elementsForTheLine[1]].clear();
		cout << elementsForTheLine[0] << " " << elementsForTheLine[1] << " " << currentDocID << endl;
	}


	/*
	cout << "*********************web page content begin" << endl;
	for(int tempCounter = 0; tempCounter < buf_len; tempCounter++){
		cout << buf[tempCounter];
	}
	cout << "*********************web page content End." << endl;
	*/

	//cout << "*********************parsing begin" << endl;

	const char* curr_p = buf;

    assert(buf != NULL);
    assert(buf_len > 0);
    uint32_t doc_id = 0;

    Tag tag_ret;  // The special type of tag we encountered.
    unsigned char context = '\0';  // Bit array for the context.
    uint32_t position = 0;         // Tracks position of each word, final position for a document is it's size in words.

    // For parsing HTML.
    bool in_closing_tag = false;  // True when we're parsing a closing tag.
    bool in_script = false;       // True when we're parsing contents of script tag.

    // For TREC documents.
    bool in_doc = false;     // True when we're parsing contents of doc tag.
    bool in_docno = false;   // True when we're parsing contents of docno tag.
    bool in_dochdr = false;  // True when we're parsing contents of dochdr tag.

    // Track the starting point of various things we want to parse out.
    const char* word_p;        // Standalone word.
    // For clueweb data ONLY, currently. I comment them out currently.
    //const char* url_p;         // TREC document URL.
    //const char* docno_p;       // TREC document number.
    const char* tag_p = NULL;  // Tracks the starting point of a tag; doubles as a flag as to whether we're currently in a tag.


    //variables set but currently not used.
    if(false){
    	cout << in_doc << endl;
    	cout << in_docno << endl;
    	cout << in_dochdr << endl;
    }

    // The main parsing loop
    while (IsWithinBounds(curr_p, buf, buf_len)) {
      if (!IsIndexable(*curr_p)) {
        if (*curr_p != '>') {
          if (*curr_p == '<') {
            tag_p = curr_p;
          }
          ++curr_p;
          continue;
        }

        if (!tag_p) {
          ++curr_p;
          continue;
        }

        // At this point, we must have just seen the end of a closing tag, '>'.
        ++curr_p;
        tag_ret = ProcessTag(tag_p, curr_p - tag_p, in_closing_tag, doc_id);

        switch (tag_ret) {
          case kTagNot:
            break;

          case kTagB:
            UpdateContext(context, in_closing_tag, kContextB);
            break;

          case kTagI:
            UpdateContext(context, in_closing_tag, kContextI);
            break;

          case kTagH:
            UpdateContext(context, in_closing_tag, kContextH);
            break;

          case kTagTitle:
            UpdateContext(context, in_closing_tag, kContextT);
            break;

          case kTagScript:
            in_script = in_closing_tag ? false : true;
            break;

          //If this tag is to be </Html>.
          case kTagHtml:
            if (doc_type_ != kWarc)
              break;

            if (in_closing_tag) {
              in_doc = false;



              break;
            }

          //If this tag is to be </Doc>.
          case kTagDoc:
            if (doc_type_ != kWarc)
              break;

            if (in_closing_tag) {
              in_doc = false;







            }
            else {
              in_doc = true;
            }
            break;

          case kTagDocno:
            if (doc_type_ != kTrec)
              break;

            in_docno = in_closing_tag ? false : true;
            break;

          case kTagDochdr:
            if (doc_type_ != kTrec)
              break;

            in_dochdr = in_closing_tag ? false : true;
            break;

          default:
            break;
        }

        tag_p = NULL;
        continue;
      }
      // Ignore everything between <script></script> tags and ignore inner contents of tags.
      if (in_script || tag_p) {
        ++curr_p;
        continue;
      }
      word_p = curr_p;
      while (IsWithinBounds(curr_p, buf, buf_len) && IsIndexable(*curr_p)) {
        ++curr_p;
      }

      //callback_->ProcessTerm(word_p, curr_p - word_p, doc_id, position++, context);
      currentWord = "";
      for(int tempCounter = 0; tempCounter < curr_p - word_p; tempCounter++){
    	  currentWord += word_p[tempCounter];
      }

      if(lookUpTermDocIDDict.count(currentWord) > 0){
    	  //cout << currentWord << " " << position << " " << int(context) << endl;
    	  outputDict[ lookUpTermDocIDDict[currentWord] + " " + currentWord].push_back(position++);
    	  outputDict[ lookUpTermDocIDDict[currentWord] + " " + currentWord].push_back(int(context));
      }
      else{
    	  position++;
      }
    }
    //cout << "*********************parsing end." << endl;
    cout << "*********************outputDict results shown begin." << endl;
    for(outputDictIter=outputDict.begin(); outputDictIter != outputDict.end(); outputDictIter++){
    	cout << (*outputDictIter).first << " " << currentDocID << " " << (*outputDictIter).second.size() << " ";
    	outputFileHandler << (*outputDictIter).first << " " << currentDocID << " " << (*outputDictIter).second.size() << " ";
    	for(unsigned int tempCounter = 0; tempCounter < (*outputDictIter).second.size(); tempCounter++){
    		cout << (*outputDictIter).second[tempCounter] << " ";
    		outputFileHandler << (*outputDictIter).second[tempCounter] << " ";
    	}
    	cout << endl;
    	outputFileHandler << endl;
    }
    cout << "*********************outputDict results shown end." << endl;
	return 0;
}

template<class Callback>
  int Parser<Callback>::ShowContentAndSimpleParse(const char* buf, int buf_len){

	map<string, string> lookUpTermDocIDDict;
	map<string, vector<int> > outputDict;
	map<string, vector<int> >::iterator outputDictIter;
	string currentWord = "";

	cout << "*********************web page content begin" << endl;
	for(int tempCounter = 0; tempCounter < buf_len; tempCounter++){
		cout << buf[tempCounter];
	}
	cout << "*********************web page content End." << endl;

	cout << "*********************parsing begin" << endl;

	const char* curr_p = buf;

    assert(buf != NULL);
    assert(buf_len > 0);
    uint32_t doc_id = 0;

    Tag tag_ret;  // The special type of tag we encountered.
    unsigned char context = '\0';  // Bit array for the context.
    uint32_t position = 0;         // Tracks position of each word, final position for a document is it's size in words.

    // For parsing HTML.
    bool in_closing_tag = false;  // True when we're parsing a closing tag.
    bool in_script = false;       // True when we're parsing contents of script tag.

    // For TREC documents.
    bool in_doc = false;     // True when we're parsing contents of doc tag.
    bool in_docno = false;   // True when we're parsing contents of docno tag.
    bool in_dochdr = false;  // True when we're parsing contents of dochdr tag.

    // Track the starting point of various things we want to parse out.
    const char* word_p;        // Standalone word.
    // For clueweb data ONLY, currently. I comment them out currently.
    //const char* url_p;         // TREC document URL.
    //const char* docno_p;       // TREC document number.
    const char* tag_p = NULL;  // Tracks the starting point of a tag; doubles as a flag as to whether we're currently in a tag.


    //variables set but currently not used.
    if(false){
    	cout << in_doc << endl;
    	cout << in_docno << endl;
    	cout << in_dochdr << endl;
    }

    // The main parsing loop
    while (IsWithinBounds(curr_p, buf, buf_len)) {
      if (!IsIndexable(*curr_p)) {
        if (*curr_p != '>') {
          if (*curr_p == '<') {
            tag_p = curr_p;
          }
          ++curr_p;
          continue;
        }

        if (!tag_p) {
          ++curr_p;
          continue;
        }

        // At this point, we must have just seen the end of a closing tag, '>'.
        ++curr_p;
        tag_ret = ProcessTag(tag_p, curr_p - tag_p, in_closing_tag, doc_id);

        switch (tag_ret) {
          case kTagNot:
            break;

          case kTagB:
            UpdateContext(context, in_closing_tag, kContextB);
            break;

          case kTagI:
            UpdateContext(context, in_closing_tag, kContextI);
            break;

          case kTagH:
            UpdateContext(context, in_closing_tag, kContextH);
            break;

          case kTagTitle:
            UpdateContext(context, in_closing_tag, kContextT);
            break;

          case kTagScript:
            in_script = in_closing_tag ? false : true;
            break;

          //If this tag is to be </Html>.
          case kTagHtml:
            if (doc_type_ != kWarc)
              break;

            if (in_closing_tag) {
              in_doc = false;



              break;
            }

          //If this tag is to be </Doc>.
          case kTagDoc:
            if (doc_type_ != kWarc)
              break;

            if (in_closing_tag) {
              in_doc = false;
            }
            else {
              in_doc = true;
            }
            break;

          case kTagDocno:
            if (doc_type_ != kTrec)
              break;

            in_docno = in_closing_tag ? false : true;
            break;

          case kTagDochdr:
            if (doc_type_ != kTrec)
              break;

            in_dochdr = in_closing_tag ? false : true;
            break;

          default:
            break;
        }

        tag_p = NULL;
        continue;
      }
      // Ignore everything between <script></script> tags and ignore inner contents of tags.
      if (in_script || tag_p) {
        ++curr_p;
        continue;
      }
      word_p = curr_p;
      while (IsWithinBounds(curr_p, buf, buf_len) && IsIndexable(*curr_p)) {
        ++curr_p;
      }

      //callback_->ProcessTerm(word_p, curr_p - word_p, doc_id, position++, context);
      currentWord = "";
      for(int tempCounter = 0; tempCounter < curr_p - word_p; tempCounter++){
    	  currentWord += word_p[tempCounter];
      }

      if(lookUpTermDocIDDict.count(currentWord) > 0){
    	  //cout << currentWord << " " << position << " " << int(context) << endl;
    	  outputDict[ lookUpTermDocIDDict[currentWord] + " " + currentWord].push_back(position++);
    	  outputDict[ lookUpTermDocIDDict[currentWord] + " " + currentWord].push_back(int(context));
      }
      else{
    	  position++;
      }
    }

    cout << "*********************parsing end." << endl;
	return 0;
}

// Returns the number of documents parsed if parsing mode is set to 'MANY_DOC', otherwise 0.
// TODO: The base URL can be set by a <base> tag within the page and by the Content-Location field in the web server's HTTP response header.
//        These cases are not currently covered.
template<class Callback>
  int Parser<Callback>::ParseDocumentCollectionAndExtractingInfoForPhase2Pruning(const char* buf, int buf_len, uint32_t& doc_id, int& avg_doc_length) {
    assert(buf != NULL);
    assert(buf_len > 0);

    int num_docs_parsed = 0;
    const char* curr_p = buf;  // Tracks the current point in the buffer.

    if (doc_type_ == kWarc){
    	std::cout << "Not Supportive Doc Type for the clueweb09." << std::endl;
    }
    else if (doc_type_ == kTrec){
        // old and working version but there are some bugs(Updated by Wei 2013/07/14)
    	// num_docs_parsed += ParseTrecDataBufferForPhase2Pruning(buf, buf_len, doc_id, avg_doc_length, curr_p);
    	// current version in development(Updated by Wei 2013/07/14)
    	// Updated by Wei on 2013/11/02 night at school
    	// NOT Used since 2013/11/04 afternoon at school because I want to implement selected parsing for the documents
    	// This is working but NOT fitted for selected parsing
    	// num_docs_parsed += ParseTrecDataBufferForPhase2PruningNewTryUpdatedOn20130714Night(buf, buf_len, doc_id, avg_doc_length, curr_p, true);

    	// This is my trying to implement version which has the function of doing selected parsing.
    	num_docs_parsed += ParseTrecDataBufferForSelectedParsingUpdatedOn20131104Afternoon(buf, buf_len, doc_id, avg_doc_length, curr_p, true);
    }
    else{
        //No Supportive doc_type_
        std::cout << "Not Supportive Doc Type." << std::endl;
    }
    return num_docs_parsed;
  }


// Returns the number of documents parsed if parsing mode is set to 'MANY_DOC', otherwise 0.
// TODO: The base URL can be set by a <base> tag within the page and by the Content-Location field in the web server's HTTP response header.
//        These cases are not currently covered.
template<class Callback>
  int Parser<Callback>::ParseDocumentCollection(const char* buf, int buf_len, uint32_t& doc_id, int& avg_doc_length) {
    assert(buf != NULL);
    assert(buf_len > 0);

    // for debug ONLY
    // std::cout << "ParseDocumentCollection(...) called." << std::endl;

    int num_docs_parsed = 0;

    const char* curr_p = buf;  // Tracks the current point in the buffer.

    if (doc_type_ == kWarc){
        // The WARC format starts each bundle with 6 info lines, which we skip here.
        // the WARC-Type is:warcinfo

        WarcHeader warc_header;

        curr_p += ProcessWarcInfoHeader(buf, buf_len, curr_p, &warc_header);

        curr_p += warc_header.content_length;

        while (true) {
          // If parsing the WARC format, need to process the header first for each document.
          // This warc header type is:response

          int header_bytes = ProcessWarcResponseAndHTTPHeader(buf, buf_len, curr_p, &warc_header, doc_id);

          if (header_bytes == 0)
            break;

          //original ver.
          curr_p += header_bytes;
          const char* content_start = curr_p;


		  num_docs_parsed += ParseWarcDataBuffer(content_start, warc_header.content_length, doc_id, avg_doc_length, curr_p, &warc_header);

		  // Since we're parsing one document at a time, we need to update the docID count, average doc length here.
		  ++doc_id;
		  ++num_docs_parsed;

		  //std::cout << "num WARC docs parsed: " << num_docs_parsed << std::endl;

		  if ((curr_p - content_start) != warc_header.content_length) {
			assert(false);
		  }


        }
    }
    else if (doc_type_ == kTrec)
    {
        //Supportive doc_type_ Trec
        num_docs_parsed += ParseTrecDataBuffer(buf, buf_len, doc_id, avg_doc_length, curr_p);
        //  for debug ONLY
        /*
        std::cout << "num_docs_parsed(NEW):" << num_docs_parsed << std::endl;
        */
    }
    else{
        //No Supportive doc_type_
        std::cout << "Not Supportive Doc Type, please wait for the update" << std::endl;
    }
    return num_docs_parsed;
  }

// Returns the number of documents parsed if parsing mode is set to 'MANY_DOC', otherwise 0.
// TODO: The base URL can be set by a <base> tag within the page and by the Content-Location field in the web server's HTTP response header.
//        These cases are not currently covered.
template<class Callback>
  int Parser<Callback>::ParseDocumentCollection(const char* buf, int buf_len, uint32_t& doc_id, int& avg_doc_length,map<string, int> &priorityLookUpDict, map<string, long> &lookUpDict, map<string, string> &lookUpDictForPopularSite) {
    assert(buf != NULL);
    assert(buf_len > 0);

    int num_docs_parsed = 0;

    const char* curr_p = buf;  // Tracks the current point in the buffer.

    if (doc_type_ == kWarc){
      // The WARC format starts each bundle with 6 info lines, which we skip here.
      // the WARC-Type is:warcinfo

      WarcHeader warc_header;
      curr_p += ProcessWarcInfoHeader(buf, buf_len, curr_p, &warc_header);
      curr_p += warc_header.content_length;


      map<string, int>::iterator priorityLookUpDictIter;
      map<string, long>::iterator lookUpDictIter;
      map<string, string>::iterator lookUpDictIterForPopularSite;



      string outputDocumentResultFileName = Configuration::GetResultValue( Configuration::GetConfiguration().GetStringValue(config_properties::kDocumentFeatureOutputFilePath));
      ofstream outputfileForDocument( outputDocumentResultFileName.c_str(), ios::out | ios::app );


      while (true) {
    	  //std::cout << "in range1" << std::endl;
        // If using the WARC format, need to process the header first for each document.
        // Right, each document will also have a warc header.
        // This warc header type is:response
        int header_bytes = ProcessWarcResponseAndHTTPHeader(buf, buf_len, curr_p, &warc_header, doc_id, priorityLookUpDict);
        if (header_bytes == 0)
          break;

        //original ver.
        curr_p += header_bytes;
        const char* content_start = curr_p;

        if( ! warc_header.fastPrioritySkip){
        	num_docs_parsed += ParseWarcDataBuffer(content_start, warc_header.content_length, doc_id, avg_doc_length, curr_p, &warc_header);

        	// Since we're parsing one document at a time, we need to update the docID count, average doc length here.
            ++doc_id;
            ++num_docs_parsed;

            // Write the features into the feature file.
            /******************************************************************************************************************************/
            //Here, let's grab the spam value from the file.
            vector<string> elementsForLookUpString;
            split(elementsForLookUpString, warc_header.feature_WARC_TREC_ID, is_any_of("-") );
            string searchString = elementsForLookUpString[1] + "-" + elementsForLookUpString[2];
            //cout << "search string:" << searchString << endl;

    		if(lookUpDict.find(searchString) != lookUpDict.end()){
    			lookUpDictIter = lookUpDict.find(searchString);
    			//cout << "Find Entry:" << lookUpDictIter->second << endl;
    			warc_header.feature_killing_spam_score = getHamProbablity(lookUpDictIter->second , warc_header.feature_WARC_TREC_ID);
    		}
    		else{
    			cout << "Can not find the probability value for:" << warc_header.feature_WARC_TREC_ID << endl;
    			warc_header.feature_killing_spam_score = -1;
    		}
            /******************************************************************************************************************************/


    		/******************************************************************************************************************************/
    		//Here,let's figure it out whether this url(web page) is popular or not.

    		//cout << "the host name I want is:" << warc_header.host_name << endl;
    		//cout << "simple test" << endl;
    		//cout << lookUpDictForPopularSite.count("google.com");

    		if ( lookUpDictForPopularSite.count(warc_header.host_name) > 0){
    			warc_header.feature_isPopular = true;
    		}
    		else{
    			warc_header.feature_isPopular = false;
    		}
    		/******************************************************************************************************************************/

    		//TODO output url as well
            outputfileForDocument << warc_header.feature_WARC_TREC_ID << " " << warc_header.urlInStringFormat << " "<< warc_header.feature_doc_id << " " << warc_header.feature_killing_spam_score << " " << warc_header.feature_isPopular << " " << warc_header.feature_urlLength << " " << warc_header.feature_documentSize << " "<< warc_header.feature_numberOfWordsInDoc << " " << warc_header.feature_numberOfDistantWordsInDoc << endl;
            //std::cout << "num WARC docs parsed: " << num_docs_parsed << std::endl;

            if ((curr_p - content_start) != warc_header.content_length) {
              assert(false);
            }
        }
        else{
        	//Wei: Let's give it a try.
        	curr_p += warc_header.content_length;
        }
      }

      outputfileForDocument.close();
    }
    else if (doc_type_ == kTrec)
    {
        //Supportive doc_type_ Trec
        num_docs_parsed += ParseTrecDataBuffer(buf, buf_len, doc_id, avg_doc_length, curr_p);
    }
    else{
        //No Supportive doc_type_
        std::cout << "Not Supportive Doc Type, please wait for the update" << std::endl;
    }
    return num_docs_parsed;
  }

template<class Callback>
  int Parser<Callback>::getHamProbablity(long offset, string WARCTRECIDNeededToLookUp){
		  string dataFileName = Configuration::GetResultValue( Configuration::GetConfiguration().GetStringValue(config_properties::kSpamRankFile));
	  	  ifstream myinputFileActualLookUp(dataFileName.c_str());
	  	  string line = "";
	  	  int percentageValue = -1;

	  	  //TODO
		  //currently no ending bound check and that is not good.
		  myinputFileActualLookUp.seekg(offset, ios::beg);
		  //cout << "offset:" << offset << endl;
		  //cout << "WARCTRECIDNeededToLookUp:" << WARCTRECIDNeededToLookUp << endl;


		  while ( myinputFileActualLookUp.good() )
		  {

			  //Do a sequential scan here.
			  getline (myinputFileActualLookUp,line);

			  if(line != ""){
				  trim(line);
				  //cout << "line:" << line << endl;


				  vector<string> elementsForGoal1;
				  vector<string> elementsForGoal2;

				  split(elementsForGoal1, line, is_any_of(" ") );
				  split(elementsForGoal2, elementsForGoal1[1], is_any_of("-") );

				  string currentMatchingString = elementsForGoal2[0] + "-" + elementsForGoal2[1] + "-" + elementsForGoal2[2] + "-" + elementsForGoal2[3];

				  //cout << "currentMatchingString:" << currentMatchingString << endl;
				  //cout << "lookUpString:" << lookUpString << endl;
				  if (currentMatchingString == WARCTRECIDNeededToLookUp){
					  //cout << "WARC_TREC_ID:" << WARCTRECIDNeededToLookUp << endl;
					  //cout << "ham probability:" << elementsForGoal1[0] << "%" << endl;
					  percentageValue = atoi( elementsForGoal1[0].c_str() );
					  break;
				  }
			  }
		  }
		  myinputFileActualLookUp.close();
		  return percentageValue;
}

template<class Callback>
  int Parser<Callback>::ParseWarcDataBuffer(const char* buf, int buf_len, uint32_t& doc_id, int& avg_doc_length, const char*& curr_p, WarcHeader* header) {
	//std::cout << "Process Web Pages Data Buffer(WARC format)" << std::endl;
	int wordCounter = 0;

	string currentWord;
	map<string, int> wordLookUpDictForEachDocument;
	map<string, int>::iterator wordUpDictIterForEachDocument;

	// Wei: updated 2012/10/02, this is for generating the features.
    // string outputTermResultFileName = Configuration::GetResultValue( Configuration::GetConfiguration().GetStringValue(config_properties::kTermFeaturePartialOutputFilePath));
    // ofstream outputfileForTerm( outputTermResultFileName.c_str(), ios::out | ios::app);


    assert(buf != NULL);
    assert(buf_len > 0);

    uint32_t initial_doc_id = doc_id;

    Tag tag_ret;  // The special type of tag we encountered.

    unsigned char context = '\0';  // Bit array for the context.
    uint32_t position = 0;         // Tracks position of each word, final position for a document is it's size in words.

    // For parsing HTML.
    bool in_closing_tag = false;  // True when we're parsing a closing tag.
    bool in_script = false;       // True when we're parsing contents of script tag.

    // For TREC documents.
    bool in_doc = false;     // True when we're parsing contents of doc tag.
    bool in_docno = false;   // True when we're parsing contents of docno tag.
    bool in_dochdr = false;  // True when we're parsing contents of dochdr tag.

    // Track the starting point of various things we want to parse out.
    const char* word_p;        // Standalone word.
    // The following 2 lines are just for TREC data. Cause this is the parser for WARC data, I comment it out.
    //const char* url_p;         // TREC document URL.
    //const char* docno_p;       // TREC document number.
    const char* tag_p = NULL;  // Tracks the starting point of a tag; doubles as a flag as to whether we're currently in a tag.

    //The following are the variables set but not used currently.
    if(false){
    	cout << in_doc << endl;
    	cout << in_docno << endl;
    	cout << in_dochdr << endl;
    }

    // The main parsing loop
    while (IsWithinBounds(curr_p, buf, buf_len)) {
      if (!IsIndexable(*curr_p)) {
        if (*curr_p != '>') {
          if (*curr_p == '<') {
            tag_p = curr_p;
          }
          ++curr_p;
          continue;
        }

        if (!tag_p) {
          ++curr_p;
          continue;
        }

        // At this point, we must have just seen the end of a closing tag, '>'.
        ++curr_p;
        tag_ret = ProcessTag(tag_p, curr_p - tag_p, in_closing_tag, doc_id);

        switch (tag_ret) {
          case kTagNot:
            break;

          case kTagB:
            UpdateContext(context, in_closing_tag, kContextB);
            break;

          case kTagI:
            UpdateContext(context, in_closing_tag, kContextI);
            break;

          case kTagH:
            UpdateContext(context, in_closing_tag, kContextH);
            break;

          case kTagTitle:
            UpdateContext(context, in_closing_tag, kContextT);
            break;

          case kTagScript:
            in_script = in_closing_tag ? false : true;
            break;

          //If this tag is to be </Html>.
          case kTagHtml:
            if (doc_type_ != kWarc)
              break;

            if (in_closing_tag) {
              in_doc = false;
              // The position at this time is actually the document length.
              avg_doc_length += position;

              //wei: I do not think this is necessary. I have deal with it in the function called: WarcResponse Header ONLY
              //callback_->ProcessDocLength(position, doc_id);
              break;
            }

          //If this tag is to be </Doc>.
          case kTagDoc:
            if (doc_type_ != kWarc)
              break;

            if (in_closing_tag) {
              in_doc = false;

              // The position at this time is actually the document length.
              avg_doc_length += position;


              //wei: I do not think this is necessary. I have deal with it in the function called: WarcResponse Header ONLY
              //callback_->ProcessDocLength(position, doc_id);


            }
            else {
              in_doc = true;
            }
            break;

          case kTagDocno:
            if (doc_type_ != kTrec)
              break;

            in_docno = in_closing_tag ? false : true;
            break;

          case kTagDochdr:
            if (doc_type_ != kTrec)
              break;

            in_dochdr = in_closing_tag ? false : true;
            break;

          default:
            break;
        }

        tag_p = NULL;
        continue;
      }

      // Ignore everything between <script></script> tags and ignore inner contents of tags.
      if (in_script || tag_p) {
        ++curr_p;
        continue;
      }



      word_p = curr_p;
      while (IsWithinBounds(curr_p, buf, buf_len) && IsIndexable(*curr_p)) {
        ++curr_p;
      }



      //This is just for temp implementation.
      currentWord = "";
      for(int iCounter = 0; iCounter < curr_p - word_p; iCounter++){
		  //outputfileForTerm << word_p[iCounter];
      	currentWord += tolower(word_p[iCounter]);
      }
      // <=0:is not an element of mymap. >0:is an element of mymap.
      if ( wordLookUpDictForEachDocument.count(currentWord) <= 0){
    	  wordLookUpDictForEachDocument[currentWord] = 1;
      }

      // for feature generations. Temp block them.
      /*
      outputfileForTerm << currentWord;
	  outputfileForTerm << " ";
	  outputfileForTerm << doc_id;
	  outputfileForTerm << " ";
	  outputfileForTerm << position;
	  outputfileForTerm << " ";
	  outputfileForTerm << (int)context;
	  outputfileForTerm << std::endl;
      */

      // generate the term level feature
	  wordCounter ++;



      callback_->ProcessTerm(word_p, curr_p - word_p, doc_id, position++, context,false);

    }

    header->feature_numberOfWordsInDoc = wordCounter;
    header->feature_numberOfDistantWordsInDoc = wordLookUpDictForEachDocument.size();

    // outputfileForTerm.close();

    return doc_id - initial_doc_id;
  }

// Updated by Wei on 2013/11/04 afternoon at school
template<class Callback>
int Parser<Callback>::ParseTrecDataBufferForSelectedParsingUpdatedOn20131104Afternoon(const char* buf, int buf_len, uint32_t& current_doc_id, int& avg_doc_length, const char*& curr_p, bool usePriorityDocumentListFlag) {

	// for debug ONLY
	std::cout << "Updated on 2013/11/04 afternoon by Wei at school." << std::endl;
	std::cout << "Parser<Callback>::ParseTrecDataBufferForSelectedParsingUpdatedOn20131104Afternoon(...) is called." << std::endl;

	assert(buf != NULL);
	assert(buf_len > 0);

	// version in test since 2013/11/04 afternoon at school by Wei
	uint32_t num_of_docs_found_from_this_buffer = 0;
	// original version
	// uint32_t initial_doc_id = current_doc_id;

	Tag tag_ret;  // The special type of tag we encountered.

	unsigned char context = '\0';  // Bit array for the context.
	uint32_t position = 0;         // Tracks position of each word, final position for a document is it's size in words.

	// For parsing HTML.
	bool in_closing_tag = false;  // True when we're parsing a closing tag.
	bool in_script = false;       // True when we're parsing contents of script tag.

	// For TREC documents.
	bool in_doc = false;     // True when we're parsing contents of doc tag.
	bool in_docno = false;   // True when we're parsing contents of docno tag.
	bool in_dochdr = false;  // True when we're parsing contents of dochdr tag.

	// Track the starting point of various things we want to parse out.
	const char* word_p;        // Standalone word.
	const char* url_p;         // TREC document URL.
	const char* docno_p;       // TREC document number.
	string docno_p_in_string_format;	// TREC document number in string format
	const char* tag_p = NULL;  // Tracks the starting point of a tag; doubles as a flag as to whether we're currently in a tag.

	while (IsWithinBounds(curr_p, buf, buf_len)) {
	  if (!IsIndexable(*curr_p)) {
		if (*curr_p != '>') {
		  if (*curr_p == '<') {
			tag_p = curr_p;
		  }
		  ++curr_p;
		  continue;
		}

		if (!tag_p) {
		  ++curr_p;
		  continue;
		}

		// At this point, we must have just seen the end of a closing tag, '>'.
		++curr_p;

		// Updated on 2013/11/04 morning by Wei at school
		// mark0
		// original version
		tag_ret = ProcessTag(tag_p, curr_p - tag_p, in_closing_tag, current_doc_id);
		// version in test
		/*
		if (usePriorityDocumentListFlag){
			  if( priority_document_needed_to_process_dict_.count(docno_p_in_string_format) > 0 ){
				  // callback_->GetEdgeCollectionController()->setCurrDocLen(position);
				  tag_ret = ProcessTag(tag_p, curr_p - tag_p, in_closing_tag, current_doc_id);
			  }
		}
		*/

		switch (tag_ret) {
		  case kTagNot:
			break;

		  case kTagB:
			UpdateContext(context, in_closing_tag, kContextB);
			break;

		  case kTagI:
			UpdateContext(context, in_closing_tag, kContextI);
			break;

		  case kTagH:
			UpdateContext(context, in_closing_tag, kContextH);
			break;

		  case kTagTitle:
			UpdateContext(context, in_closing_tag, kContextT);
			break;

		  case kTagScript:
			in_script = in_closing_tag ? false : true;
			break;

		  case kTagDoc:
			if (doc_type_ != kTrec)
			  break;

			if (in_closing_tag) {
			  in_doc = false;

			  // The position at this time is actually the document length.
			  avg_doc_length += position;

			  std::cout << "docLength(in words): " << position << std::endl;

			  // Updated by Wei on 2013/11/02 night at school
			  // mark1.1
			  // original version
			  // callback_->GetEdgeCollectionController()->setCurrDocLen(position);
			  // callback_->ProcessDocLength(position, current_doc_id);
			  // version in test
			  if (usePriorityDocumentListFlag){
				  if( priority_document_needed_to_process_dict_.count(docno_p_in_string_format) > 0 ){
					  // callback_->GetEdgeCollectionController()->setCurrDocLen(position);
					  callback_->ProcessDocLength(position, current_doc_id);
					  ++current_doc_id;
				  }
			  }


			  // the Graph Generation component has put here and it marks the ends of parsing the whole document
			  // Updated by Wei 2013/07/14 night
			  // mark1.2
			  /*
			  if (usePriorityDocumentListFlag){
				  if( priority_document_needed_to_process_dict_.count(docno_p_in_string_format) > 0 ){
					  if (callback_->GetEdgeCollectionController()->GetProperThreshold() <= 180){
						  callback_->GetEdgeCollectionController()->GenerateDocumentPostingGraphMethod3( current_doc_id, Configuration::GetResultValue<long int>(Configuration::GetConfiguration().GetNumericalValue(config_properties::kTermPairEdgeFreqThreshold)),false, true, false, false, false, false, true );
					  }
					  else{
						  callback_->GetEdgeCollectionController()->GenerateDocumentPostingGraphMethod1( current_doc_id, Configuration::GetResultValue<long int>(Configuration::GetConfiguration().GetNumericalValue(config_properties::kTermPairEdgeFreqThreshold)),false, true, false, false, false, false, true );
					  }

				  }
				  else{
					  // should clear the variable for the current document
					  // even NOT doing the graph generation part, you still need to clear up the data structures.
					  callback_->GetEdgeCollectionController()->ClearDocIDRelatedTermDicts(current_doc_id); // erasing the whole doc by key
					  callback_->GetEdgeCollectionController()->ClearTermsBothInCurrentDocIDAndQueryTraceVector(); // clear the terms_both_in_current_doc_id_and_query_trace_vector_
				  }
				  std::cout << std::endl;
			  }
			  else{
				  if (callback_->GetEdgeCollectionController()->GetProperThreshold() <= 180){
					  callback_->GetEdgeCollectionController()->GenerateDocumentPostingGraphMethod3( current_doc_id, Configuration::GetResultValue<long int>(Configuration::GetConfiguration().GetNumericalValue(config_properties::kTermPairEdgeFreqThreshold)), false, true, false, false, false, false, true );
				  }
				  else{
					  callback_->GetEdgeCollectionController()->GenerateDocumentPostingGraphMethod1( current_doc_id, Configuration::GetResultValue<long int>(Configuration::GetConfiguration().GetNumericalValue(config_properties::kTermPairEdgeFreqThreshold)), false, true, false, false, false, false, true );
				  }
			  }
			  */

			  // This only applies when we're parsing multiple documents in one go.
			  if (parsing_mode_ == kManyDoc) {
				context = 0;
				position = 0;
				// version in test
				++num_of_docs_found_from_this_buffer;
				// original version
				// ++current_doc_id;

				// Need to reset certain properties before moving on to the next document.
				in_script = false;
			  }
			}
			else {
			  in_doc = true;
			}
			break;

		  case kTagDocno:
			if (doc_type_ != kTrec)
			  break;

			in_docno = in_closing_tag ? false : true;
			break;

		  case kTagDochdr:
			if (doc_type_ != kTrec)
			  break;

			in_dochdr = in_closing_tag ? false : true;
			break;

		  default:
			break;
		}

		tag_p = NULL;
		continue;
	  }

	  // Ignore everything between <script></script> tags and ignore inner contents of tags.
	  if (in_script || tag_p) {
		++curr_p;
		continue;
	  }

	  if (doc_type_ == kTrec)
	  {
		if (in_docno) {
		  docno_p = curr_p;
		  while (IsWithinBounds(curr_p, buf, buf_len) && *curr_p != '<') {
			++curr_p;
		  }


		  // for debug ONLY
		  docno_p_in_string_format = string(docno_p,curr_p - docno_p);

		  // NOT fitted for current version
		  // std::cout << "doc_id(internal): " << current_doc_id << std::endl;
		  std::cout << "docno(trecID): " << docno_p_in_string_format << std::endl;

		  // Updated on 2013/11/02 night by Wei at school
		  // Mark2
		  // original version
		  // callback_->GetEdgeCollectionController()->setCurrDocTrecID(docno_p_in_string_format);
		  // callback_->ProcessDocno(docno_p, curr_p - docno_p, current_doc_id);
		  // version in test
		  if (usePriorityDocumentListFlag){
			  if( priority_document_needed_to_process_dict_.count(docno_p_in_string_format) > 0 ){
				  callback_->GetEdgeCollectionController()->setCurrDocTrecID(docno_p_in_string_format);
				  callback_->ProcessDocno(docno_p, curr_p - docno_p, current_doc_id);
			  }
		  }


		  continue;
		} else if (in_dochdr) {
		  BitSet(context, kContextU);
		  url_p = curr_p;
		  while (IsWithinBounds(url_p, buf, buf_len) && *url_p != '\n') {
			if (!IsIndexable(*url_p)) {
			  url_p++;
			  continue;
			}

			word_p = url_p;
			while (IsWithinBounds(url_p, buf, buf_len) && IsIndexable(*url_p)) {
			  url_p++;
			}

			// Updated on 2013/11/02 night by Wei at school
			// Mark3
			// original version
			// callback_->ProcessTerm(word_p, url_p - word_p, current_doc_id, position++, context, false);
			// version in test
			if (usePriorityDocumentListFlag){
				if( priority_document_needed_to_process_dict_.count(docno_p_in_string_format) > 0 ){
					callback_->ProcessTerm(word_p, url_p - word_p, current_doc_id, position++, context, false);
				}
			}

		  }

		  BitUnset(context, kContextU);

		  // Updated on 2013/11/02 night by Wei at school
		  // Mark4
		  // original version
		  // callback_->ProcessUrl(curr_p, url_p - curr_p, current_doc_id);
		  // version in test
		  if (usePriorityDocumentListFlag){
			  if( priority_document_needed_to_process_dict_.count(docno_p_in_string_format) > 0 ){
				  callback_->ProcessUrl(curr_p, url_p - curr_p, current_doc_id);
			  }
		  }


		  curr_p = url_p + 1;
		  // Skip the rest of the dochdr contents (making sure that we're at the end of the dochdr).
		  while (IsWithinBounds(curr_p, buf, buf_len)) {
			if (*curr_p == '<') {
			  const char kDocHdrClosingTag[] = "</DOCHDR>";
			  // Make sure it's actually the closing tag.
			  if (IsWithinBounds(curr_p + sizeof(kDocHdrClosingTag) - 1, buf, buf_len) && strncasecmp(curr_p, kDocHdrClosingTag, sizeof(kDocHdrClosingTag) - 1) == 0) {
				break;
			  }
			}
			++curr_p;
		  }

		  continue;
		}
	  }

	  word_p = curr_p;
	  while (IsWithinBounds(curr_p, buf, buf_len) && IsIndexable(*curr_p)) {
		++curr_p;
	  }

	  // Updated on 2013/11/02 night by Wei at school
	  // Mark5
	  // original version
	  // callback_->ProcessTerm(word_p, curr_p - word_p, current_doc_id, position++, context, false);
	  // version in test
	  if (usePriorityDocumentListFlag){
		  if( priority_document_needed_to_process_dict_.count(docno_p_in_string_format) > 0 ){
			  callback_->ProcessTerm(word_p, curr_p - word_p, current_doc_id, position++, context, false);
		  }
	  }

	}
	return num_of_docs_found_from_this_buffer;
}


// Updated by Wei on 2013/11/02 night at school
// Let all the callback_ functions listens to the usePriorityDocumentListFlag.
template<class Callback>
int Parser<Callback>::ParseTrecDataBufferForPhase2PruningNewTryUpdatedOn20130714Night(const char* buf, int buf_len, uint32_t& current_doc_id, int& avg_doc_length, const char*& curr_p, bool usePriorityDocumentListFlag) {

	// for debug ONLY
	std::cout << "Updated on 2013/11/04 morning by Wei at school." << std::endl;
	std::cout << "Parser<Callback>::ParseTrecDataBufferForPhase2PruningNewTryUpdatedOn20130714Night(...) is called." << std::endl;

	assert(buf != NULL);
	assert(buf_len > 0);

	uint32_t initial_doc_id = current_doc_id;

	Tag tag_ret;  // The special type of tag we encountered.

	unsigned char context = '\0';  // Bit array for the context.
	uint32_t position = 0;         // Tracks position of each word, final position for a document is it's size in words.

	// For parsing HTML.
	bool in_closing_tag = false;  // True when we're parsing a closing tag.
	bool in_script = false;       // True when we're parsing contents of script tag.

	// For TREC documents.
	bool in_doc = false;     // True when we're parsing contents of doc tag.
	bool in_docno = false;   // True when we're parsing contents of docno tag.
	bool in_dochdr = false;  // True when we're parsing contents of dochdr tag.

	// Track the starting point of various things we want to parse out.
	const char* word_p;        // Standalone word.
	const char* url_p;         // TREC document URL.
	const char* docno_p;       // TREC document number.
	string docno_p_in_string_format;	// TREC document number in string format
	const char* tag_p = NULL;  // Tracks the starting point of a tag; doubles as a flag as to whether we're currently in a tag.

	//this is the dummy code:
	if(false){
		 cout << in_doc << endl;
	}

	while (IsWithinBounds(curr_p, buf, buf_len)) {
	  if (!IsIndexable(*curr_p)) {
		if (*curr_p != '>') {
		  if (*curr_p == '<') {
			tag_p = curr_p;
		  }
		  ++curr_p;
		  continue;
		}

		if (!tag_p) {
		  ++curr_p;
		  continue;
		}

		// At this point, we must have just seen the end of a closing tag, '>'.
		++curr_p;

		// Updated on 2013/11/04 morning by Wei at school
		// version in test
		/*
		if (usePriorityDocumentListFlag){
			  if( priority_document_needed_to_process_dict_.count(docno_p_in_string_format) > 0 ){
				  // callback_->GetEdgeCollectionController()->setCurrDocLen(position);
				  callback_->ProcessDocLength(position, current_doc_id);
			  }
		}
		*/
		// original version
		tag_ret = ProcessTag(tag_p, curr_p - tag_p, in_closing_tag, current_doc_id);

		switch (tag_ret) {
		  case kTagNot:
			break;

		  case kTagB:
			UpdateContext(context, in_closing_tag, kContextB);
			break;

		  case kTagI:
			UpdateContext(context, in_closing_tag, kContextI);
			break;

		  case kTagH:
			UpdateContext(context, in_closing_tag, kContextH);
			break;

		  case kTagTitle:
			UpdateContext(context, in_closing_tag, kContextT);
			break;

		  case kTagScript:
			in_script = in_closing_tag ? false : true;
			break;

		  case kTagDoc:
			if (doc_type_ != kTrec)
			  break;

			if (in_closing_tag) {
			  in_doc = false;

			  // The position at this time is actually the document length.
			  avg_doc_length += position;
			  /*
			  #ifdef PARSERINL_DEBUG
				std::cout << "doc_length is: " << position << std::endl;
				std::cout << "doc_id is: " << doc_id << std::endl;
			  #endif
			  */


			  std::cout << "docLength(in words): " << position << std::endl;

			  // Updated by Wei on 2013/11/02 night at school
			  // mark1
			  // original version
			  // callback_->GetEdgeCollectionController()->setCurrDocLen(position);
			  callback_->ProcessDocLength(position, current_doc_id);
			  // version in test
			  /*
			  if (usePriorityDocumentListFlag){
				  if( priority_document_needed_to_process_dict_.count(docno_p_in_string_format) > 0 ){
					  // callback_->GetEdgeCollectionController()->setCurrDocLen(position);
					  callback_->ProcessDocLength(position, current_doc_id);
				  }
			  }
			  */

			  // I should put the graph generation here because it marks the ending of parsing the whole document
			  // Updated by Wei 2013/07/14 night
			  /*
			  if (usePriorityDocumentListFlag){
				  if( priority_document_needed_to_process_dict_.count(docno_p_in_string_format) > 0 ){
					  if (callback_->GetEdgeCollectionController()->GetProperThreshold() <= 180){
						  callback_->GetEdgeCollectionController()->GenerateDocumentPostingGraphMethod3( current_doc_id, Configuration::GetResultValue<long int>(Configuration::GetConfiguration().GetNumericalValue(config_properties::kTermPairEdgeFreqThreshold)),false, true, false, false, false, false, true );
					  }
					  else{
						  callback_->GetEdgeCollectionController()->GenerateDocumentPostingGraphMethod1( current_doc_id, Configuration::GetResultValue<long int>(Configuration::GetConfiguration().GetNumericalValue(config_properties::kTermPairEdgeFreqThreshold)),false, true, false, false, false, false, true );
					  }

				  }
				  else{
					  // should clear the variable for the current document
					  // even NOT doing the graph generation part, you still need to clear up the data structures.
					  callback_->GetEdgeCollectionController()->ClearDocIDRelatedTermDicts(current_doc_id); // erasing the whole doc by key
					  callback_->GetEdgeCollectionController()->ClearTermsBothInCurrentDocIDAndQueryTraceVector(); // clear the terms_both_in_current_doc_id_and_query_trace_vector_
				  }
				  std::cout << std::endl;
			  }
			  else{
				  if (callback_->GetEdgeCollectionController()->GetProperThreshold() <= 180){
					  callback_->GetEdgeCollectionController()->GenerateDocumentPostingGraphMethod3( current_doc_id, Configuration::GetResultValue<long int>(Configuration::GetConfiguration().GetNumericalValue(config_properties::kTermPairEdgeFreqThreshold)), false, true, false, false, false, false, true );
				  }
				  else{
					  callback_->GetEdgeCollectionController()->GenerateDocumentPostingGraphMethod1( current_doc_id, Configuration::GetResultValue<long int>(Configuration::GetConfiguration().GetNumericalValue(config_properties::kTermPairEdgeFreqThreshold)), false, true, false, false, false, false, true );
				  }
			  }
			  */

			  // This only applies when we're parsing multiple documents in one go.
			  if (parsing_mode_ == kManyDoc) {
				context = 0;
				position = 0;
				++current_doc_id;

				// Need to reset certain properties before moving on to the next document.
				in_script = false;
			  }
			}
			else {
			  in_doc = true;
			}
			break;

		  case kTagDocno:
			if (doc_type_ != kTrec)
			  break;

			in_docno = in_closing_tag ? false : true;
			break;

		  case kTagDochdr:
			if (doc_type_ != kTrec)
			  break;

			in_dochdr = in_closing_tag ? false : true;
			break;

		  default:
			break;
		}

		tag_p = NULL;
		continue;
	  }

	  // Ignore everything between <script></script> tags and ignore inner contents of tags.
	  if (in_script || tag_p) {
		++curr_p;
		continue;
	  }

	  if (doc_type_ == kTrec)
	  {
		if (in_docno) {
		  docno_p = curr_p;
		  while (IsWithinBounds(curr_p, buf, buf_len) && *curr_p != '<') {
			++curr_p;
		  }


		  // for debug ONLY
		  docno_p_in_string_format = string(docno_p,curr_p - docno_p);

		  std::cout << "doc_id(internal): " << current_doc_id << std::endl;
		  std::cout << "docno(trecID): " << docno_p_in_string_format << std::endl;

		  // Updated on 2013/11/02 night by Wei at school
		  // Mark2
		  // original version
		  // callback_->GetEdgeCollectionController()->setCurrDocTrecID(docno_p_in_string_format);
		  callback_->ProcessDocno(docno_p, curr_p - docno_p, current_doc_id);
		  // version in test
		  /*
		  if (usePriorityDocumentListFlag){
			  if( priority_document_needed_to_process_dict_.count(docno_p_in_string_format) > 0 ){
				  callback_->GetEdgeCollectionController()->setCurrDocTrecID(docno_p_in_string_format);
				  callback_->ProcessDocno(docno_p, curr_p - docno_p, current_doc_id);
			  }
		  }
		  */


		  continue;
		} else if (in_dochdr) {
		  BitSet(context, kContextU);

		  url_p = curr_p;
		  while (IsWithinBounds(url_p, buf, buf_len) && *url_p != '\n') {
			if (!IsIndexable(*url_p)) {
			  url_p++;
			  continue;
			}

			word_p = url_p;
			while (IsWithinBounds(url_p, buf, buf_len) && IsIndexable(*url_p)) {
			  url_p++;
			}

			// Updated on 2013/11/02 night by Wei at school
			// Mark3
			// original version
			callback_->ProcessTerm(word_p, url_p - word_p, current_doc_id, position++, context, false);
			// version in test
			/*
			if (usePriorityDocumentListFlag){
				if( priority_document_needed_to_process_dict_.count(docno_p_in_string_format) > 0 ){
					callback_->ProcessTerm(word_p, url_p - word_p, current_doc_id, position++, context, false);
				}
			}
			*/
		  }

		  BitUnset(context, kContextU);
		  /*
		  #ifdef PARSERINL_DEBUG
			std::cout << "url is: " << curr_p << std::endl;
			std::cout << "url_len is: " << url_p - curr_p << std::endl;
			std::cout << "doc_id is: " << doc_id << std::endl;
		  #endif
		  */

		  // Updated on 2013/11/02 night by Wei at school
		  // Mark4
		  // original version
		  callback_->ProcessUrl(curr_p, url_p - curr_p, current_doc_id);
		  // version in test
		  /*
		  if (usePriorityDocumentListFlag){
			  if( priority_document_needed_to_process_dict_.count(docno_p_in_string_format) > 0 ){
				  callback_->ProcessUrl(curr_p, url_p - curr_p, current_doc_id);
			  }
		  }
		  */
		  curr_p = url_p + 1;
		  // Skip the rest of the dochdr contents (making sure that we're at the end of the dochdr).
		  while (IsWithinBounds(curr_p, buf, buf_len)) {
			if (*curr_p == '<') {
			  const char kDocHdrClosingTag[] = "</DOCHDR>";
			  // Make sure it's actually the closing tag.
			  if (IsWithinBounds(curr_p + sizeof(kDocHdrClosingTag) - 1, buf, buf_len) && strncasecmp(curr_p, kDocHdrClosingTag, sizeof(kDocHdrClosingTag) - 1) == 0) {
				break;
			  }
			}
			++curr_p;
		  }

		  continue;
		}
	  }

	  word_p = curr_p;
	  while (IsWithinBounds(curr_p, buf, buf_len) && IsIndexable(*curr_p)) {
		++curr_p;
	  }

	  // Updated on 2013/11/02 night by Wei at school
	  // Mark5
	  // original version
	  callback_->ProcessTerm(word_p, curr_p - word_p, current_doc_id, position++, context, false);
	  // version in test
	  /*
	  if (usePriorityDocumentListFlag){
		  if( priority_document_needed_to_process_dict_.count(docno_p_in_string_format) > 0 ){
			  callback_->ProcessTerm(word_p, curr_p - word_p, current_doc_id, position++, context, false);
		  }
	  }
	  */
	}
	return current_doc_id - initial_doc_id;
}

/*
template<class Callback>
  int Parser<Callback>::ParseTrecDataBufferForPhase2Pruning(const char* buf, int buf_len, uint32_t& current_doc_id, int& avg_doc_length, const char*& curr_p) {

  // for debug ONLY
  // std::cout << "ParseTrecDataBufferForPhase2Pruning(...) is called." << std::endl;

  assert(buf != NULL);
  assert(buf_len > 0);

  uint32_t initial_doc_id = current_doc_id;

  // Updated by Wei 2013/03/13 afternoon
  uint32_t trigger_doc_id = initial_doc_id;
  bool trigger_doc_id_be_updated = false;

  Tag tag_ret;  // The special type of tag we encountered.

  unsigned char context = '\0';  // Bit array for the context.
  uint32_t position = 0;         // Tracks position of each word, final position for a document is it's size in words.

  // For parsing HTML.
  bool in_closing_tag = false;  // True when we're parsing a closing tag.
  bool in_script = false;       // True when we're parsing contents of script tag.

  // For TREC documents.
  bool in_doc = false;     // True when we're parsing contents of doc tag.
  bool in_docno = false;   // True when we're parsing contents of docno tag.
  bool in_dochdr = false;  // True when we're parsing contents of dochdr tag.

  // Track the starting point of various things we want to parse out.
  const char* word_p;        // Standalone word.
  const char* url_p;         // TREC document URL.
  const char* docno_p;       // TREC document number.
  string docno_p_in_string_format;	// TREC document number in string format
  const char* tag_p = NULL;  // Tracks the starting point of a tag; doubles as a flag as to whether we're currently in a tag.

  //this is the dummy code:
  if(false){
	 cout << in_doc << endl;
  }

  while (IsWithinBounds(curr_p, buf, buf_len)) {
    if (!IsIndexable(*curr_p)) {
      if (*curr_p != '>') {
        if (*curr_p == '<') {
          tag_p = curr_p;
        }
        ++curr_p;
        continue;
      }

      if (!tag_p) {
        ++curr_p;
        continue;
      }

      // At this point, we must have just seen the end of a closing tag, '>'.
      ++curr_p;
      tag_ret = ProcessTag(tag_p, curr_p - tag_p, in_closing_tag, current_doc_id);

      switch (tag_ret) {
        case kTagNot:
          break;

        case kTagB:
          UpdateContext(context, in_closing_tag, kContextB);
          break;

        case kTagI:
          UpdateContext(context, in_closing_tag, kContextI);
          break;

        case kTagH:
          UpdateContext(context, in_closing_tag, kContextH);
          break;

        case kTagTitle:
          UpdateContext(context, in_closing_tag, kContextT);
          break;

        case kTagScript:
          in_script = in_closing_tag ? false : true;
          break;

        case kTagDoc:
          if (doc_type_ != kTrec)
            break;

          if (in_closing_tag) {
            in_doc = false;

            // The position at this time is actually the document length.
            avg_doc_length += position;

            // #ifdef PARSERINL_DEBUG
            //  std::cout << "doc_length is: " << position << std::endl;
            //  std::cout << "doc_id is: " << doc_id << std::endl;
            // #endif


            // for debug ONLY 1
            // std::cout << "setCurrDocLen test in..." << std::endl;
            callback_->GetEdgeCollectionController()->setCurrDocLen(position);
            // for debug ONLY 2
            // std::cout << "setCurrDocLen test out." << std::endl;

            callback_->ProcessDocLength(position, current_doc_id);

            // This only applies when we're parsing multiple documents in one go.
            if (parsing_mode_ == kManyDoc) {
              context = 0;
              position = 0;
              ++current_doc_id;

              // Need to reset certain properties before moving on to the next document.
              in_script = false;
            }
          }
          else {
            in_doc = true;
          }
          break;

        case kTagDocno:
          if (doc_type_ != kTrec)
            break;

          in_docno = in_closing_tag ? false : true;
          break;

        case kTagDochdr:
          if (doc_type_ != kTrec)
            break;

          in_dochdr = in_closing_tag ? false : true;
          break;

        default:
          break;
      }

      tag_p = NULL;
      continue;
    }

    // Ignore everything between <script></script> tags and ignore inner contents of tags.
    if (in_script || tag_p) {
      ++curr_p;
      continue;
    }

    if (doc_type_ == kTrec)
    {
      if (in_docno) {
        docno_p = curr_p;
        while (IsWithinBounds(curr_p, buf, buf_len) && *curr_p != '<') {
          ++curr_p;
        }


        // for debug ONLY
        docno_p_in_string_format = string(docno_p,curr_p - docno_p);

        std::cout << "doc_id(internal): " << current_doc_id << std::endl;
        std::cout << "docno(trecID): " << docno_p_in_string_format << std::endl;

        callback_->GetEdgeCollectionController()->setCurrDocTrecID(docno_p_in_string_format);
        callback_->ProcessDocno(docno_p, curr_p - docno_p, current_doc_id);

        continue;
      } else if (in_dochdr) {
        BitSet(context, kContextU);

        url_p = curr_p;
        while (IsWithinBounds(url_p, buf, buf_len) && *url_p != '\n') {
          if (!IsIndexable(*url_p)) {
            url_p++;
            continue;
          }

          word_p = url_p;
          while (IsWithinBounds(url_p, buf, buf_len) && IsIndexable(*url_p)) {
            url_p++;
          }

          // fill the value for the trigger_doc_id
          if( priority_document_needed_to_process_dict_.count(docno_p_in_string_format) > 0 ){
        	  trigger_doc_id = current_doc_id;
        	  trigger_doc_id_be_updated = true;
          }

          if( (current_doc_id - trigger_doc_id) == 1 and trigger_doc_id_be_updated){
        	  callback_->ProcessTerm(word_p, url_p - word_p, current_doc_id, position++, context, true);
          }
          else{
        	  callback_->ProcessTerm(word_p, url_p - word_p, current_doc_id, position++, context, false);
          }


        }

        BitUnset(context, kContextU);

        // #ifdef PARSERINL_DEBUG
        //  std::cout << "url is: " << curr_p << std::endl;
        //  std::cout << "url_len is: " << url_p - curr_p << std::endl;
        //  std::cout << "doc_id is: " << doc_id << std::endl;
        // #endif

        callback_->ProcessUrl(curr_p, url_p - curr_p, current_doc_id);

        curr_p = url_p + 1;
        // Skip the rest of the dochdr contents (making sure that we're at the end of the dochdr).
        while (IsWithinBounds(curr_p, buf, buf_len)) {
          if (*curr_p == '<') {
            const char kDocHdrClosingTag[] = "</DOCHDR>";
            // Make sure it's actually the closing tag.
            if (IsWithinBounds(curr_p + sizeof(kDocHdrClosingTag) - 1, buf, buf_len) && strncasecmp(curr_p, kDocHdrClosingTag, sizeof(kDocHdrClosingTag) - 1) == 0) {
              break;
            }
          }
          ++curr_p;
        }

        continue;
      }
    }

    word_p = curr_p;
    while (IsWithinBounds(curr_p, buf, buf_len) && IsIndexable(*curr_p)) {
      ++curr_p;
    }

    // fill the value for the trigger_doc_id
    if( priority_document_needed_to_process_dict_.count(docno_p_in_string_format) > 0){
  	  trigger_doc_id = current_doc_id;
  	  trigger_doc_id_be_updated = true;
    }

    if( (current_doc_id - trigger_doc_id) == 1 and trigger_doc_id_be_updated){
    	callback_->ProcessTerm(word_p, curr_p - word_p, current_doc_id, position++, context, true);
    }
    else{
    	callback_->ProcessTerm(word_p, curr_p - word_p, current_doc_id, position++, context, false);
    }

  }


  // generate the last document posting dependency graph :)
  // for debug ONLY 2013/03/05
  // option1: run the method1
  // callback_->GetEdgeCollectionController()->GenerateDocumentPostingGraphMethod1( doc_id-1,Configuration::GetResultValue<long int>(Configuration::GetConfiguration().GetNumericalValue(config_properties::kTermPairEdgeFreqThreshold)) );

  // option2: run the method2
  // callback_->GetEdgeCollectionController()->GenerateDocumentPostingGraphMethod2( doc_id-1,Configuration::GetResultValue<long int>(Configuration::GetConfiguration().GetNumericalValue(config_properties::kTermPairEdgeFreqThreshold)) );

  // option3: run the method3
  // callback_->GetEdgeCollectionController()->GenerateDocumentPostingGraphMethod3( doc_id-1, Configuration::GetResultValue<long int>(Configuration::GetConfiguration().GetNumericalValue(config_properties::kTermPairEdgeFreqThreshold)) );

  // option4: run the method1 and method3 together with the setting of the threshold.
  // cout << "callback_->GetEdgeCollectionController()->GetProperThreshold():" << callback_->GetEdgeCollectionController()->GetProperThreshold() << endl;
  // for comparison
  // callback_->GetEdgeCollectionController()->GenerateDocumentPostingGraphMethod1( doc_id-1, Configuration::GetResultValue<long int>(Configuration::GetConfiguration().GetNumericalValue(config_properties::kTermPairEdgeFreqThreshold)) );
  // callback_->GetEdgeCollectionController()->GenerateDocumentPostingGraphMethod3( doc_id-1, Configuration::GetResultValue<long int>(Configuration::GetConfiguration().GetNumericalValue(config_properties::kTermPairEdgeFreqThreshold)) );

  if( priority_document_needed_to_process_dict_.count(docno_p_in_string_format) > 0 ){
	  // If in the priority document list, should of course generate the graph first and then clear the data structures which used
	  if (callback_->GetEdgeCollectionController()->GetProperThreshold() <= 180){
		  callback_->GetEdgeCollectionController()->GenerateDocumentPostingGraphMethod3( current_doc_id-1, Configuration::GetResultValue<long int>(Configuration::GetConfiguration().GetNumericalValue(config_properties::kTermPairEdgeFreqThreshold)),false, true );
	  }
	  else{
		  callback_->GetEdgeCollectionController()->GenerateDocumentPostingGraphMethod1( current_doc_id-1, Configuration::GetResultValue<long int>(Configuration::GetConfiguration().GetNumericalValue(config_properties::kTermPairEdgeFreqThreshold)),false, true );
	  }
  }
  else{
	  // If NOT in the priority document list, should directly clear the data structures which used
	  // should clear the variable for the current document
	  // even NOT doing the graph generation part, you still need to clear up the data structures.
	  callback_->GetEdgeCollectionController()->ClearDocIDRelatedTermDicts( current_doc_id-1 ); // erasing the whole doc by key
	  callback_->GetEdgeCollectionController()->ClearTermsBothInCurrentDocIDAndQueryTraceVector(); // clear the terms_both_in_current_doc_id_and_query_trace_vector_
  }

  return current_doc_id - initial_doc_id;
}
*/



template<class Callback>
  int Parser<Callback>::ParseTrecDataBuffer(const char* buf, int buf_len, uint32_t& doc_id, int& avg_doc_length, const char*& curr_p) {
  assert(buf != NULL);
  assert(buf_len > 0);

  uint32_t initial_doc_id = doc_id;

  Tag tag_ret;  // The special type of tag we encountered.

  unsigned char context = '\0';  // Bit array for the context.
  uint32_t position = 0;         // Tracks position of each word, final position for a document is it's size in words.

  // For parsing HTML.
  bool in_closing_tag = false;  // True when we're parsing a closing tag.
  bool in_script = false;       // True when we're parsing contents of script tag.

  // For TREC documents.
  bool in_doc = false;     // True when we're parsing contents of doc tag.
  bool in_docno = false;   // True when we're parsing contents of docno tag.
  bool in_dochdr = false;  // True when we're parsing contents of dochdr tag.

  // Track the starting point of various things we want to parse out.
  const char* word_p;        // Standalone word.
  const char* url_p;         // TREC document URL.
  const char* docno_p;       // TREC document number.
  const char* tag_p = NULL;  // Tracks the starting point of a tag; doubles as a flag as to whether we're currently in a tag.

  //this is the dummy code:
  if(false){
	 cout << in_doc << endl;
  }

  while (IsWithinBounds(curr_p, buf, buf_len)) {
    if (!IsIndexable(*curr_p)) {
      if (*curr_p != '>') {
        if (*curr_p == '<') {
          tag_p = curr_p;
        }
        ++curr_p;
        continue;
      }

      if (!tag_p) {
        ++curr_p;
        continue;
      }

      // At this point, we must have just seen the end of a closing tag, '>'.
      ++curr_p;
      tag_ret = ProcessTag(tag_p, curr_p - tag_p, in_closing_tag, doc_id);

      switch (tag_ret) {
        case kTagNot:
          break;

        case kTagB:
          UpdateContext(context, in_closing_tag, kContextB);
          break;

        case kTagI:
          UpdateContext(context, in_closing_tag, kContextI);
          break;

        case kTagH:
          UpdateContext(context, in_closing_tag, kContextH);
          break;

        case kTagTitle:
          UpdateContext(context, in_closing_tag, kContextT);
          break;

        case kTagScript:
          in_script = in_closing_tag ? false : true;
          break;

        case kTagDoc:
          if (doc_type_ != kTrec)
            break;

          if (in_closing_tag) {
            in_doc = false;

            // The position at this time is actually the document length.
            avg_doc_length += position;
            /*
            #ifdef PARSERINL_DEBUG
              std::cout << "doc_length is: " << position << std::endl;
              std::cout << "doc_id is: " << doc_id << std::endl;
            #endif
            */
            callback_->ProcessDocLength(position, doc_id);

            // This only applies when we're parsing multiple documents in one go.
            if (parsing_mode_ == kManyDoc) {
              context = 0;
              position = 0;
              ++doc_id;

              // Need to reset certain properties before moving on to the next document.
              in_script = false;
            }
          }
          else {
            in_doc = true;
          }
          break;

        case kTagDocno:
          if (doc_type_ != kTrec)
            break;

          in_docno = in_closing_tag ? false : true;
          break;

        case kTagDochdr:
          if (doc_type_ != kTrec)
            break;

          in_dochdr = in_closing_tag ? false : true;
          break;

        default:
          break;
      }

      tag_p = NULL;
      continue;
    }

    // Ignore everything between <script></script> tags and ignore inner contents of tags.
    if (in_script || tag_p) {
      ++curr_p;
      continue;
    }

    if (doc_type_ == kTrec)
    {
      if (in_docno) {
        docno_p = curr_p;
        while (IsWithinBounds(curr_p, buf, buf_len) && *curr_p != '<') {
          ++curr_p;
        }

        /*
        #ifdef PARSERINL_DEBUG
          std::cout << "docno is: " << docno_p << std::endl;
          std::cout << "docno_len is: " << curr_p - docno_p << std::endl;
          std::cout << "doc_id is: " << doc_id << std::endl;
        #endif
        */
        callback_->ProcessDocno(docno_p, curr_p - docno_p, doc_id);

        continue;
      } else if (in_dochdr) {
        BitSet(context, kContextU);

        url_p = curr_p;
        while (IsWithinBounds(url_p, buf, buf_len) && *url_p != '\n') {
          if (!IsIndexable(*url_p)) {
            url_p++;
            continue;
          }

          word_p = url_p;
          while (IsWithinBounds(url_p, buf, buf_len) && IsIndexable(*url_p)) {
            url_p++;
          }

          //IndexingParserCallback::ProcessTerm called(), amazing and I do not know why?
          // std::cout << "test2 in" << std::endl;
          callback_->ProcessTerm(word_p, url_p - word_p, doc_id, position++, context, true);
          // std::cout << "test2 out" << std::endl;
        }

        BitUnset(context, kContextU);
        /*
        #ifdef PARSERINL_DEBUG
          std::cout << "url is: " << curr_p << std::endl;
          std::cout << "url_len is: " << url_p - curr_p << std::endl;
          std::cout << "doc_id is: " << doc_id << std::endl;
        #endif
        */
        callback_->ProcessUrl(curr_p, url_p - curr_p, doc_id);

        curr_p = url_p + 1;
        // Skip the rest of the dochdr contents (making sure that we're at the end of the dochdr).
        while (IsWithinBounds(curr_p, buf, buf_len)) {
          if (*curr_p == '<') {
            const char kDocHdrClosingTag[] = "</DOCHDR>";
            // Make sure it's actually the closing tag.
            if (IsWithinBounds(curr_p + sizeof(kDocHdrClosingTag) - 1, buf, buf_len) && strncasecmp(curr_p, kDocHdrClosingTag, sizeof(kDocHdrClosingTag) - 1) == 0) {
              break;
            }
          }
          ++curr_p;
        }

        continue;
      }
    }

    word_p = curr_p;
    while (IsWithinBounds(curr_p, buf, buf_len) && IsIndexable(*curr_p)) {
      ++curr_p;
    }
    // std::cout << "test3 in" << std::endl;
    callback_->ProcessTerm(word_p, curr_p - word_p, doc_id, position++, context, true);
    // std::cout << "test3 out" << std::endl;
  }


  // **********************************Not Used Anymore since 2013/07/09 night***************************************************
  // generate the last document posting dependency graph :)
  // for debug ONLY 2013/03/05
  // option1: run the method1
  // callback_->GetEdgeCollectionController()->GenerateDocumentPostingGraphMethod1( doc_id-1,Configuration::GetResultValue<long int>(Configuration::GetConfiguration().GetNumericalValue(config_properties::kTermPairEdgeFreqThreshold)) );

  // option2: run the method2
  // callback_->GetEdgeCollectionController()->GenerateDocumentPostingGraphMethod2( doc_id-1,Configuration::GetResultValue<long int>(Configuration::GetConfiguration().GetNumericalValue(config_properties::kTermPairEdgeFreqThreshold)) );

  // option3: run the method3
  // callback_->GetEdgeCollectionController()->GenerateDocumentPostingGraphMethod3( doc_id-1, Configuration::GetResultValue<long int>(Configuration::GetConfiguration().GetNumericalValue(config_properties::kTermPairEdgeFreqThreshold)) );

  // option4: run the method1 and method3 together with the setting of the threshold.
  // cout << "callback_->GetEdgeCollectionController()->GetProperThreshold():" << callback_->GetEdgeCollectionController()->GetProperThreshold() << endl;
  // for comparison
  // callback_->GetEdgeCollectionController()->GenerateDocumentPostingGraphMethod1( doc_id-1, Configuration::GetResultValue<long int>(Configuration::GetConfiguration().GetNumericalValue(config_properties::kTermPairEdgeFreqThreshold)) );
  // callback_->GetEdgeCollectionController()->GenerateDocumentPostingGraphMethod3( doc_id-1, Configuration::GetResultValue<long int>(Configuration::GetConfiguration().GetNumericalValue(config_properties::kTermPairEdgeFreqThreshold)) );
  // ****************************************************************************************************************************

  // ***********************************current used version since 2013/07/09*******************************
  // Logic begins...
  // Block the following function for parsing speedup (for Shi's experiment)

  if (callback_->GetEdgeCollectionController()->GetProperThreshold() <= 180){
	  callback_->GetEdgeCollectionController()->GenerateDocumentPostingGraphMethod3( doc_id-1, Configuration::GetResultValue<long int>(Configuration::GetConfiguration().GetNumericalValue(config_properties::kTermPairEdgeFreqThreshold)),false, true, false, false, false, false, true );
  }
  else{
	  callback_->GetEdgeCollectionController()->GenerateDocumentPostingGraphMethod1( doc_id-1, Configuration::GetResultValue<long int>(Configuration::GetConfiguration().GetNumericalValue(config_properties::kTermPairEdgeFreqThreshold)),false, true, false, false, false, false, true );
  }

  // Logic ends.
  // ***********************************current used version since 2013/07/09*******************************

  return doc_id - initial_doc_id;
}

template<class Callback>
  typename Parser<Callback>::Tag Parser<Callback>::ProcessTag(const char* tag, int tag_len, bool& in_closing_tag, uint32_t doc_id) {
    // Caller must ensure tag_len is always at least 2, for tag "<>".
    assert(tag_len >= 2);

    const char* curr_tag_p = tag + 1;

    // Check whether this is a closing tag.
    if (*curr_tag_p == '/') {
      in_closing_tag = true;
      ++curr_tag_p;
    } else {
      in_closing_tag = false;
    }

    switch (*curr_tag_p) {
      case 'a':
      case 'A': {
        const char* l_start = NULL;
        const char* l_end = NULL;

        for (++curr_tag_p; IsWithinBounds(curr_tag_p, tag, tag_len); ++curr_tag_p) {
          if (*curr_tag_p == '"' || *curr_tag_p == '\'') {
            if (!l_start) {
              l_start = curr_tag_p + 1;
            } else {
              l_end = curr_tag_p;
              break;
            }
          }
        }

        if (l_start && l_end) {
        	callback_->ProcessLink(l_start, l_end - l_start, doc_id);
        }

        return Parser::kTagNot;
      }
      case 'b':
      case 'B': {
        return IsValidTag(curr_tag_p, tag, tag_len, "b") ? Parser::kTagB : Parser::kTagNot;
      }
      case 'i':
      case 'I': {
        return IsValidTag(curr_tag_p, tag, tag_len, "i") ? Parser::kTagI : Parser::kTagNot;
      }
      case 'e':
      case 'E': {
        return IsValidTag(curr_tag_p, tag, tag_len, "em") ? Parser::kTagI : Parser::kTagNot;
      }
      case 'h':
      case 'H':
      {
        //Original Ver. Too beautiful, wei want to modify it.
        //return AreValidTags(curr_tag_p, tag, tag_len, "h", "1", "6", 2) ? Parser::kTagH : Parser::kTagNot;

        //Wei Ver.
        return AreValidTags(curr_tag_p, tag, tag_len, "h", "1", "6", 2) ? Parser::kTagH : IsValidTag(curr_tag_p, tag, tag_len, "html") ? Parser::kTagHtml : Parser::kTagNot;
      }
      case 't':
      case 'T': {
        return IsValidTag(curr_tag_p, tag, tag_len, "title") ? Parser::kTagTitle : Parser::kTagNot;
      }

      case 's':
      case 'S': {
        return IsValidTag(curr_tag_p, tag, tag_len, "strong") ? Parser::kTagB : IsValidTag(curr_tag_p, tag, tag_len, "script") ? Parser::kTagScript
                                                                                                                               : Parser::kTagNot;
      }
      case 'd':
      case 'D': {
        return IsValidTag(curr_tag_p, tag, tag_len, "dochdr") ? Parser::kTagDochdr
                                                              : IsValidTag(curr_tag_p, tag, tag_len, "docno") ? Parser::kTagDocno
                                                                                                              : IsValidTag(curr_tag_p, tag, tag_len, "doc") ? Parser::kTagDoc
                                                                                                                                                            : Parser::kTagNot;
      }
      default: {
        break;
      }
    }

    return Parser::kTagNot;
  }

template<class Callback>
  bool Parser<Callback>::IsValidTag(const char* curr_tag_p, const char* tag, int tag_len, const char curr_tag_name[]) {
    const size_t curr_tag_name_len = strlen(curr_tag_name);

    if (!strncasecmp(curr_tag_p, curr_tag_name, curr_tag_name_len)) {
      curr_tag_p += curr_tag_name_len;
      for (; IsWithinBounds(curr_tag_p, tag, tag_len); ++curr_tag_p) {
        if (*curr_tag_p != ' ') {
          if (*curr_tag_p == '>')
            return true;
          else
            break;
        }
      }
    }

    return false;
  }

template<class Callback>
  inline bool Parser<Callback>::IsWithinBounds(const char* curr, const char* start, int len) {
    return (curr - start) < len;
  }

template<class Callback>
  bool Parser<Callback>::AreValidTags(const char* curr_tag_p, const char* tag, int tag_len, const char curr_tag_base[], const char start_range[],
                                      const char end_range[], int range_len) {
    const size_t curr_tag_base_len = strlen(curr_tag_base);

    if (!strncasecmp(curr_tag_p, curr_tag_base, curr_tag_base_len)) {
      curr_tag_p += curr_tag_base_len;

      if (IsWithinBounds(curr_tag_p + range_len - 1, tag, tag_len)) {
        if (strncmp(curr_tag_p, start_range, range_len) >= 0 && strncmp(curr_tag_p, end_range, range_len) <= 0) {
          curr_tag_p += range_len;

          for (; IsWithinBounds(curr_tag_p, tag, tag_len); ++curr_tag_p) {
            if (*curr_tag_p != ' ') {
              if (*curr_tag_p == '>')
                return true;
              else
                break;
            }
          }
        }
      }
    }

    return false;
  }

template<class Callback>
  int Parser<Callback>::ProcessWarcInfoHeader(const char* buf, int buf_len, const char* curr_p, WarcHeader* header) {
      std::cout << "Process ProcessWarcInfoHeader (The same)" << std::endl;
      const char* header_start = curr_p;

      while (IsWithinBounds(curr_p, buf, buf_len)) {
        if (*curr_p == '\n') {
          ++curr_p;

          // Double new line means we read the complete header.
          if (IsWithinBounds(curr_p, buf, buf_len) && *curr_p == '\n') {
            ++curr_p;
            break;
          }

          const char* key_start = curr_p;
          int key_length = 0;
          while (IsWithinBounds(curr_p, buf, buf_len) && *curr_p != ':') {
            ++curr_p;
            ++key_length;
          }

          #ifdef PARSERINL_DEBUG
            std::string warc_key = std::string(key_start, key_length);
            //std::cout << "WARC key: " << warc_key << std::endl;
          #endif



          ++curr_p;

          if (IsWithinBounds(curr_p, buf, buf_len) && *curr_p == ' ') {
            ++curr_p;
          }

          const char* value_start = curr_p;
          int value_length = 0;

          while (IsWithinBounds(curr_p, buf, buf_len) && *curr_p != '\n') {
            ++curr_p;
            ++value_length;
          }

          const char kContentLengthStr[] = "Content-Length";
          const int kContentLengthStrLen = sizeof(kContentLengthStr) - 1;

          #ifdef PARSERINL_DEBUG
            std::string warc_value = std::string(value_start, value_length);
            //std::cout << "WARC value: " << warc_value << std::endl;
          #endif

          // We need to know how long the document content is.
          if (key_length == kContentLengthStrLen && strncmp(key_start, kContentLengthStr, kContentLengthStrLen) == 0) {
            char content_length_buf[11];  // Has to fit a 4 byte integer and a terminating null character.
            assert(value_length < 11);
            memcpy(content_length_buf, value_start, value_length);
            content_length_buf[value_length] = '\0';
            header->content_length = atoi(content_length_buf);
          }
        } else {
          ++curr_p; // Skips past the 'WARC/0.18' part.
        }
      }

      return (curr_p - header_start);
  }


template<class Callback>
	string Parser<Callback>::parseURLForDomainName(string inputURL){
		string line = "";
		string x = "";
		string host_name = "";
		trim(inputURL);
		x = inputURL;
		size_t sp = x.find_first_of( '/', 7 /* skip http:// part */ );
		if ( sp != string::npos ) {
			string base_url( x.begin()+7, x.begin()+sp );
			//cout << base_url << endl;
			if (boost::starts_with(base_url, "www.")){
				host_name = base_url.substr(4,base_url.size());
				//cout << host_name << endl;
			}
			else{
				host_name = base_url;
				//cout << host_name << endl;
			}
			sp = x.find_last_of( '/' );
			if ( sp != string::npos ) {
				string query( x.begin()+sp+1, x.end() );
				//cout << query << endl;
			}
		}
		return host_name;
	}


template<class Callback>
  int Parser<Callback>::ProcessWarcResponseAndHTTPHeader(const char* buf, int buf_len, const char* curr_p, WarcHeader* header, uint32_t& doc_id, map<string, int> &priorityLookUpDict) {
      const char* header_start = curr_p;
      //std::cout << "Process WarcResponse Header ONLY(not include the HTTP Header)" << std::endl;

      header->fastPrioritySkip = true;

      std::string specialStrHostName = "";
      std::string specialStrFeatureDomainName = "";
      int specialintFeatureURLLength = 0;
      int specialintFeatureNumberNestingLevels = 0;
      std::string specialStrURLInStringFormat = "";
      const char* specialCharURL = "";
      int specialCharURLLen = 0;

      uint32_t specialDocID = -1;
      std::string specialStrFeatureWARCTRECID = "";
      const char* specialCharDocno = "";
      int specialDocno_len = 0;

      std::string specialFeatureDocumentSize = "";
      int specialContentLength = 0;

      while (IsWithinBounds(curr_p, buf, buf_len))
      {


        if (*curr_p == '\n')
        {
          ++curr_p;

          // Double new line means we read the complete header.
          if (IsWithinBounds(curr_p, buf, buf_len) && *curr_p == '\n') {
            ++curr_p;
            break;
          }

          const char* key_start = curr_p;
          int key_length = 0;
          while (IsWithinBounds(curr_p, buf, buf_len) && *curr_p != ':') {
            ++curr_p;
            ++key_length;
          }


          std::string warc_key = std::string(key_start, key_length);



          ++curr_p;

          if (IsWithinBounds(curr_p, buf, buf_len) && *curr_p == ' ') {
            ++curr_p;
          }

          const char* value_start = curr_p;
          int value_length = 0;

          while (IsWithinBounds(curr_p, buf, buf_len) && *curr_p != '\n') {
            ++curr_p;
            ++value_length;
          }

           std::string warc_value = std::string(value_start, value_length);

           if(warc_key == "WARC-Target-URI")
           {

        	   //std::cout << "in the WARC-Target-URI,shit?" << std::endl;
                //This is just for temp implementation.
                //I can not make this so ugly.
        	    //TODO: find a good url parser. But I do not want to do this now.

        	    /*
            	vector<string> strs;
            	vector<string> elements;
            	split(strs, warc_value, is_any_of("/") );
            	split(elements, strs[2], is_any_of(".") );

            	//assume that elements[1] + elements[2] is with the popular site format.
            	header->host_name = elements[1] + "." + elements[2];

            	header->feature_domainName = elements[2];
            	header->feature_urlLength = warc_value.size();

            	header->feature_number_nesting_levels = strs.size() -3;
*/
        	    specialStrHostName = parseURLForDomainName(warc_value);
        	    specialStrFeatureDomainName = "NULL";
        	    specialintFeatureURLLength = warc_value.size();
        	    specialintFeatureNumberNestingLevels = 3;
        	    specialStrURLInStringFormat = warc_value;
        	    specialCharURL = warc_value.c_str();
        	    specialCharURLLen = warc_value.length();

        	    /*
            	header->host_name = specialStrHostName;
            	header->feature_domainName = specialStrFeatureDomainName;
            	header->feature_urlLength = specialintFeatureURLLength;
            	header->feature_number_nesting_levels = specialintFeatureNumberNestingLevels;
            	header->urlInStringFormat = specialStrURLInStringFormat;
            	header->url = specialCharURL;
            	header->url_len = specialCharURLLen;
            	callback_->ProcessUrl(header->url, header->url_len, doc_id);
            	*/

           }
           else if(warc_key == "WARC-TREC-ID")
           {
        	  if(priorityLookUpDict.size() != 0){
            	  //This document needed to be parsed because it is very important.
            	  if(priorityLookUpDict.count(warc_value) > 0){
            		  cout << "	Processing:" << warc_value << endl;
            		  header->fastPrioritySkip = false;
            		  priorityLookUpDict.erase(warc_value);

            	  }
            	  //This document is not very important for us until we get all the evaluated web pages.
            	  else{
            		 header->fastPrioritySkip = true;
            		 //cout << "	Processing:" << warc_value << endl;
            		 cout << "	Skipping..:" << warc_value << endl;
            	  }
        	  }
        	  else{
        		  // the size of the priorityLookUpDict has been 0, which means that all the relevent documents have been found.
        		  // It is time to do processing for every document in order to satisfy the requirement of min number of documents to be parsed.
        		  cout << "	Processing:" << warc_value << endl;
        		  header->fastPrioritySkip = false;
        	  }

        	  specialDocID = doc_id;
        	  specialStrFeatureWARCTRECID = warc_value;
        	  specialCharDocno = warc_value.c_str();
        	  specialDocno_len = warc_value.length();

           }
           else if(warc_key == "Content-Length")
           {
        	  specialFeatureDocumentSize = warc_value;
        	  specialContentLength = atoi(warc_value.c_str());
           }

        }
        else
        {
          ++curr_p; // Skips past the 'WARC/0.18' part. Correct, this operation also gives me some hint.
        }
      }
  	  header->host_name = specialStrHostName;
  	  header->feature_domainName = specialStrFeatureDomainName;
  	  header->feature_urlLength = specialintFeatureURLLength;
  	  header->feature_number_nesting_levels = specialintFeatureNumberNestingLevels;
  	  header->urlInStringFormat = specialStrURLInStringFormat;
  	  header->url = specialCharURL;
  	  header->url_len = specialCharURLLen;

  	  header->feature_doc_id = specialDocID;
  	  header->feature_WARC_TREC_ID = specialStrFeatureWARCTRECID;
      header->docno = specialCharDocno;
      header->docno_len = specialDocno_len;

	  header->feature_documentSize = specialFeatureDocumentSize;
      header->content_length = specialContentLength;

      if( ! header->fastPrioritySkip){
      	  callback_->ProcessUrl(header->url, header->url_len, doc_id);
          callback_->ProcessDocno(header->docno, header->docno_len, doc_id);
          callback_->ProcessDocLength(header->content_length, doc_id);
      }
      return (curr_p - header_start);
  }

template<class Callback>
  int Parser<Callback>::ProcessWarcResponseAndHTTPHeader(const char* buf, int buf_len, const char* curr_p, WarcHeader* header, uint32_t& doc_id) {
      const char* header_start = curr_p;
      //std::cout << "Process WarcResponse Header ONLY(not include the HTTP Header)" << std::endl;

      header->fastPrioritySkip = false;

      std::string specialStrHostName = "";
      std::string specialStrFeatureDomainName = "";
      int specialintFeatureURLLength = 0;
      int specialintFeatureNumberNestingLevels = 0;
      std::string specialStrURLInStringFormat = "";
      const char* specialCharURL = "";
      int specialCharURLLen = 0;

      uint32_t specialDocID = -1;
      std::string specialStrFeatureWARCTRECID = "";
      const char* specialCharDocno = "";
      int specialDocno_len = 0;

      std::string specialFeatureDocumentSize = "";
      int specialContentLength = 0;

      while (IsWithinBounds(curr_p, buf, buf_len))
      {


        if (*curr_p == '\n')
        {
          ++curr_p;

          // Double new line means we read the complete header.
          if (IsWithinBounds(curr_p, buf, buf_len) && *curr_p == '\n') {
            ++curr_p;
            break;
          }

          const char* key_start = curr_p;
          int key_length = 0;
          while (IsWithinBounds(curr_p, buf, buf_len) && *curr_p != ':') {
            ++curr_p;
            ++key_length;
          }


          std::string warc_key = std::string(key_start, key_length);



          ++curr_p;

          if (IsWithinBounds(curr_p, buf, buf_len) && *curr_p == ' ') {
            ++curr_p;
          }

          const char* value_start = curr_p;
          int value_length = 0;

          while (IsWithinBounds(curr_p, buf, buf_len) && *curr_p != '\n') {
            ++curr_p;
            ++value_length;
          }

           std::string warc_value = std::string(value_start, value_length);

           if(warc_key == "WARC-Target-URI")
           {

        	   //std::cout << "in the WARC-Target-URI,shit?" << std::endl;
                //This is just for temp implementation.
                //I can not make this so ugly.
        	    //TODO: find a good url parser. But I do not want to do this now.

        	    /*
            	vector<string> strs;
            	vector<string> elements;
            	split(strs, warc_value, is_any_of("/") );
            	split(elements, strs[2], is_any_of(".") );

            	//assume that elements[1] + elements[2] is with the popular site format.
            	header->host_name = elements[1] + "." + elements[2];

            	header->feature_domainName = elements[2];
            	header->feature_urlLength = warc_value.size();

            	header->feature_number_nesting_levels = strs.size() -3;
*/
        	    specialStrHostName = parseURLForDomainName(warc_value);
        	    specialStrFeatureDomainName = "NULL";
        	    specialintFeatureURLLength = warc_value.size();
        	    specialintFeatureNumberNestingLevels = 3;
        	    specialStrURLInStringFormat = warc_value;
        	    specialCharURL = warc_value.c_str();
        	    specialCharURLLen = warc_value.length();

        	    /*
            	header->host_name = specialStrHostName;
            	header->feature_domainName = specialStrFeatureDomainName;
            	header->feature_urlLength = specialintFeatureURLLength;
            	header->feature_number_nesting_levels = specialintFeatureNumberNestingLevels;
            	header->urlInStringFormat = specialStrURLInStringFormat;
            	header->url = specialCharURL;
            	header->url_len = specialCharURLLen;
            	callback_->ProcessUrl(header->url, header->url_len, doc_id);
            	*/

           }
           else if(warc_key == "WARC-TREC-ID")
           {
			  cout << "	Processing:" << warc_value << endl;
			  header->fastPrioritySkip = false;
        	  specialDocID = doc_id;
        	  specialStrFeatureWARCTRECID = warc_value;
        	  specialCharDocno = warc_value.c_str();
        	  specialDocno_len = warc_value.length();
           }
           else if(warc_key == "Content-Length")
           {
        	  specialFeatureDocumentSize = warc_value;
        	  specialContentLength = atoi(warc_value.c_str());
           }

        }
        else
        {
          ++curr_p; // Skips past the 'WARC/0.18' part. Correct, this operation also gives me some hint.
        }
      }
  	  header->host_name = specialStrHostName;
  	  header->feature_domainName = specialStrFeatureDomainName;
  	  header->feature_urlLength = specialintFeatureURLLength;
  	  header->feature_number_nesting_levels = specialintFeatureNumberNestingLevels;
  	  header->urlInStringFormat = specialStrURLInStringFormat;
  	  header->url = specialCharURL;
  	  header->url_len = specialCharURLLen;

  	  header->feature_doc_id = specialDocID;
  	  header->feature_WARC_TREC_ID = specialStrFeatureWARCTRECID;
      header->docno = specialCharDocno;
      header->docno_len = specialDocno_len;

	  header->feature_documentSize = specialFeatureDocumentSize;
      header->content_length = specialContentLength;

      if( ! header->fastPrioritySkip){
      	  callback_->ProcessUrl(header->url, header->url_len, doc_id);
          callback_->ProcessDocno(header->docno, header->docno_len, doc_id);
          callback_->ProcessDocLength(header->content_length, doc_id);
      }
      return (curr_p - header_start);
  }

#endif /* PARSERINL_H_ */
